{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "from tqdm import tqdm"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "test_data_path = '../data/A_testData0531.csv'\n",
    "train_gps_path = '../data/train0523.csv'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "141it [13:05,  7.24s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "                       0                         2           3          4   \\\n",
      "140103814  CI928033126705  2020-04-28T04:38:50.000Z  114.266082  22.569507   \n",
      "140104400  CI928033126705  2020-04-28T04:42:08.000Z  114.272042  22.564355   \n",
      "140105038  CI928033126705  2020-04-28T04:44:27.000Z  114.278100  22.562850   \n",
      "140105564  CI928033126705  2020-04-28T04:46:27.000Z  114.284892  22.562940   \n",
      "140106056  CI928033126705  2020-04-28T04:48:58.000Z  114.294103  22.563118   \n",
      "...                   ...                       ...         ...        ...   \n",
      "140603541  CI928033126705  2020-04-30T01:34:30.000Z  120.080553  22.616520   \n",
      "140604062  CI928033126705  2020-04-30T01:36:34.000Z  120.074552  22.620547   \n",
      "140604469  CI928033126705  2020-04-30T01:38:25.000Z  120.069318  22.624008   \n",
      "140605093  CI928033126705  2020-04-30T01:41:49.000Z  120.061243  22.631462   \n",
      "140605264  CI928033126705  2020-04-30T01:43:00.000Z  120.058635  22.634353   \n",
      "\n",
      "           6            12  \n",
      "140103814   8  CNYTN-MXZLO  \n",
      "140104400  15  CNYTN-MXZLO  \n",
      "140105038  17  CNYTN-MXZLO  \n",
      "140105564  20  CNYTN-MXZLO  \n",
      "140106056  22  CNYTN-MXZLO  \n",
      "...        ..          ...  \n",
      "140603541  20  CNYTN-MXZLO  \n",
      "140604062  21  CNYTN-MXZLO  \n",
      "140604469  21  CNYTN-MXZLO  \n",
      "140605093  21  CNYTN-MXZLO  \n",
      "140605264  21  CNYTN-MXZLO  \n",
      "\n",
      "[573 rows x 6 columns]\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "148it [13:50,  5.61s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "                       0                         2           3          4   \\\n",
      "147618287  CI928033126705  2020-04-28T04:38:50.000Z  114.266082  22.569507   \n",
      "147618848  CI928033126705  2020-04-28T04:42:08.000Z  114.272042  22.564355   \n",
      "147619473  CI928033126705  2020-04-28T04:44:27.000Z  114.278100  22.562850   \n",
      "147619991  CI928033126705  2020-04-28T04:46:27.000Z  114.284892  22.562940   \n",
      "147620448  CI928033126705  2020-04-28T04:48:58.000Z  114.294103  22.563118   \n",
      "...                   ...                       ...         ...        ...   \n",
      "147906482  CI928033126705  2020-04-29T06:59:59.000Z  120.330932  22.550058   \n",
      "147906820  CI928033126705  2020-04-29T07:03:05.000Z  120.330965  22.550027   \n",
      "147906965  CI928033126705  2020-04-29T07:05:56.000Z  120.330948  22.550062   \n",
      "147907369  CI928033126705  2020-04-29T07:11:58.000Z  120.330917  22.550023   \n",
      "147907512  CI928033126705  2020-04-29T07:15:07.000Z  120.330920  22.550032   \n",
      "\n",
      "           6            12  \n",
      "147618287   8  CNYTN-MXZLO  \n",
      "147618848  15  CNYTN-MXZLO  \n",
      "147619473  17  CNYTN-MXZLO  \n",
      "147619991  20  CNYTN-MXZLO  \n",
      "147620448  22  CNYTN-MXZLO  \n",
      "...        ..          ...  \n",
      "147906482   0  CNYTN-MXZLO  \n",
      "147906820   0  CNYTN-MXZLO  \n",
      "147906965   0  CNYTN-MXZLO  \n",
      "147907369   0  CNYTN-MXZLO  \n",
      "147907512   0  CNYTN-MXZLO  \n",
      "\n",
      "[345 rows x 6 columns]\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "train_data_origin_chunk = pd.read_csv(train_gps_path, chunksize = 1000000, usecols = [0,2,3,4,6,12], header=None)\n",
    "for chunk in tqdm(train_data_origin_chunk):\n",
    "#     valid_route = chunk[chunk[12].apply(lambda x: ~pd.isnull(x) and str(x).startswith(\"HKHKG\") and (\"FRFOS\" in str(x)))]\n",
    "    valid_route = chunk[chunk[0] == \"CI928033126705\"]\n",
    "    if (valid_route.size > 0):\n",
    "        print(valid_route)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3.7.6 64-bit ('AI': conda)",
   "language": "python",
   "name": "python37664bitaiconda6859e03b37c34f0182c9bde8073269f7"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}
